

import requests
from bs4 import BeautifulSoup
import time
import sys

import re
from pihsdk import pihsdk



urlPat = re.compile('\A(.*?)://')


def getItem(element, source, baseUrl):
    resItem = {}
    resItem['link_title'] =  element.get_text().strip()
    #resItem['created_at'] =  time.time()
    curUrl = element.get('href').strip()
    if curUrl[0] == '/':
        curUrl = baseUrl + curUrl[1:]
    match = urlPat.match(curUrl)
    if match:
        if match.group(1) == 'http':
            resItem['http_head'] = 0
        else:
            resItem['http_head'] = 1
        resItem['url'] = curUrl[len(match.group()):]
    else:
        return None
    resItem['source'] = source
    #resItem['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
    #resItem['status'] = 0
    
    if resItem['link_title'] == '':
        return None
    
    if 'index' in resItem['url']:
        return None
    if not 'html' in resItem['url']:
        return None
    return resItem



def main_handler(event, context):

    insert_list = []

    tarUrl = "http://www.people.com.cn/"
    header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',}
    response = requests.get(tarUrl, headers=header)
    response.encoding = "gbk"

    content = response.text

    soup = BeautifulSoup(content, "html.parser")

    news_list = soup.select('#hotnews h1 a')
    if len(news_list) > 0:
        insert_list.append(getItem(news_list[0], "people:main:title", tarUrl))

        
    insert_list = [item for item in insert_list if item is not None]
    
    if len(insert_list) > 0:
        tmp_client = pihsdk('password','fass_url_crawler','http://10.0.24.6:25000')
        return tmp_client.send_url(insert_list)
    
    
    #for item in insert_list:
        #print(item)
    #if len(insert_list) > 0:
        #resItem = {'status': 1, 'ts': int(time.time()), 'len': len(insert_list), 'data': insert_list}
        #tmp_client = pihsdk('test','http://127.0.0.1:25000')
        #print(tmp_client.send_post('/',resItem))



